/**
 * Desc:ETL服务基类
 * Author：Kevin
 * CreatedOn: 2021/4/21
 * Version: 1.0
 */

const request = require('request');
const object2csv = require('objects-to-csv');
const fs = require('fs');
const path = require('path');
const sha256 = require('sha256');

const REQUEST_QUEUE = {};

/**
 * 数据抓取和转换服务的基类
 */
class EtlServiceBase {
    /**
     * 构造函数
     * @param {string} dataRoot 数据文件存放位置
     */
    constructor(dataRoot) {
            this.params = {};
            // 响应数据列表对应的元素名称
            this.dataTag = 'list';
            // 数据文件存放位置
            this.dataRoot = dataRoot || './data';
            // 是否将请求的数据追加到文件末尾
            this.append = false;
            // 服务标志
            this.id = '_';
            // 请求数据的URL，在子类中指定
            this.url = '';
            // 没有数据目录时默认创建目录
            var fp = path.join(__dirname, this.dataRoot);
            if (!fs.existsSync(fp)) {
                console.info(`[info] create directory ${fp}.`);
                fs.mkdirSync(fp, { recursive: true });
            }
        }
        /**
         * put request params into queue
         * @returns {object} request
         */
    __create__() {
            var opt = {
                ps: this.params,
                url: this.url
            };
            var id = sha256(JSON.stringify(opt));
            var req = REQUEST_QUEUE[id];

            if (!req) {
                var ps = [`_=${Date.now()}`];
                for (var p in this.params) {
                    ps.push(`${p}=${this.params[p]}`);
                }
                req = { id: id, url: `${this.url}?${ps.join('&')}`, times: 0 };
            }

            req.times++;
            REQUEST_QUEUE[id] = req;

            return req;
        }
        /**
         * remove request params from queue
         * @param {object} req 
         */
    __done__(req, cb) {
            delete REQUEST_QUEUE[req.id];
            cb && cb();
        }
        /**
         * 执行一次http的get请求
         * @returns Promise
         */
    __get__() {
        var cb = (resolve, reject) => {
            var req = this.__create__();
            console.debug(`[info] request [${req.id}] ${req.url}`);
            request.get({
                url: req.url,
                timeout: 5000,
                headers: {
                    'UserAgent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36 Edg/90.0.818.42',
                }
            }, async(err, res) => {
                console.debug(`[info] read response...`);
                if (err) {
                    if (req.times > 9) {
                        console.error(`[error] request over ${req.times} times.`);
                        return reject(err);
                    }
                    console.error(`[error] request retry on ${req.times} times.`);
                    return await new Promise(s => setTimeout(() => {
                        cb(resolve, reject)
                    }, 5000 * Math.pow(req.times, 2)))
                }
                this.__done__(req, () => {
                    console.debug(`[info] request completed.`);
                    resolve(res);
                });
            });
        };
        return new Promise(cb);
    }

    /**
     * 对响应文本进行解析
     * @param {string} 响应请求获得的文本内容 
     * @returns {object} 解析后的json对象
     */
    __parse__(text) {

        var res = {};
        try {
            res = JSON.parse(text);
        } catch {
            var arr = text.match(/(?<=\().*?(?=\);)/);
            res = JSON.parse(arr[0]);
        }

        res.objects = [];
        if (res.data) {
            // 非数组
            if (this.dataTag == '') {
                res.objects.push(this.__object__(res.data));
            }
            // 数组
            else {
                var items = res.data[this.dataTag];
                if (items && items.length > 0) res.objects = items.map(l => this.__object__(l));
            }
        } else {
            res.objects.push(res);
        }
        return res;
    }

    /**
     * 将原始数据转换为标准的对象
     * @param {object} line 原始数据对象或字符串内容
     * @returns {object} 返回数据对象
     */
    __object__(line) {
        var o = {};
        if (typeof(line) == 'object') {
            for (var f in this.fields) {
                o[this.fields[f]] = line[f];
            }
        } else {
            var i = 0;
            var arr = line.toString().split(',');
            for (var f in this.fields) {
                o[this.fields[f]] = arr[i++];
            }
        }
        return o;
    }

    /**
     * 请求并解析响应结果
     * @returns Promise
     */
    __request__() {
        return new Promise((resolve, reject) => {
            this.__get__(this.url).then((res) => {
                console.info(`[info] [${res.statusCode}] ${res.statusMessage}.`);
                if (res.statusCode !== 200) return reject(res.statusMessage);
                var result = this.__parse__(res.body);
                console.info(`[info] parse content of response done.`);
                resolve(result);
            }).catch(reject);
        });
    }

    /**
     * 启动一次数据抓取任务
     * @param {object} params 请求参数包，可以在每次请求之前进行设置
     * @param {function} cb 回调函数，用以处理数据转换和保持等任务
     */
    start(params = {}, cb) {
        Object.assign(this.params, params);
        this.__request__().then(res => {
            cb && cb(this, res);
            let csv = new object2csv(res.objects.filter(o => !o.isn));
            csv.toDisk(`${this.dataRoot}/${this.id}.csv`, { append: this.append });
            console.info(`[info] file ${this.id}.csv saved.`);
        }).catch(err => {
            console.error(err);
        });
    }
}

module.exports = EtlServiceBase;